
package com.shujia.spark.opt

import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo9Bro {
  def main(args: Array[String]): Unit = {
    /**
     * 广播变量
     *
     */

    val conf = new SparkConf()

    conf.setMaster("local")

    conf.setAppName("bro")

    val sc = new SparkContext(conf)

    val ids = List("1500100974", "1500100992", "1500100997", "1500100995")

    //将一个变量广播出去
    //1、在Driver段定义广播变量
    val briList: Broadcast[List[String]] = sc.broadcast(ids)

    val studentsRDD: RDD[String] = sc.textFile("data/students.txt")

    val filterRDD: RDD[String] = studentsRDD.filter(line => {
      val id: String = line.split(",")(0)
      //2、在Executpr段获取广播变量
      val list: List[String] = briList.value
      //哦按段学号是否在集合中存在
      list.contains(id)
    })

    filterRDD.foreach(println)


  }

}
